import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import re
import csv

def get_all_urls(base_url):
    response = requests.get(base_url)
    response.raise_for_status()
    soup = BeautifulSoup(response.content, "html.parser")
    urls = []
    for link in soup.find_all("a", href=True):
        href = link["href"]
        absolute_url = urljoin(base_url, href)
        urls.append(absolute_url)
    return urls

def extract_text_from_url(url):
    response = requests.get(url)
    response.raise_for_status()
    soup = BeautifulSoup(response.content, "html.parser")
    text = soup.get_text()
    text = re.sub(r'\n+', '\n', text)
    text = re.sub(r'\s+', ' ', text)
    return text


base_urls = [
    "https://www.haigan.gr.jp/public/guidebook/2024/"

]


all_urls = []
for base_url in base_urls:
    all_urls.extend(get_all_urls(base_url))

print("Number of extracted URLs:", len(all_urls))
print("List of extracted URLs:")
for url in all_urls:
    print(url)